package com.kaowen.spider;

import java.awt.geom.GeneralPath;
import java.awt.geom.Point2D;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.ArrayListHandler;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;

import com.google.gson.Gson;
import com.kaowen.spider.util.DBManager;

/**
 * 网络爬虫 百度地图搜索 关键字纵深查询 +地毯式搜索
 * 
 * @author thinkpad
 */
public class SpiderForBaidu {

	private static Logger logger = Logger.getLogger(SpiderForBaidu.class);
	private Connection conn = null;
	private Gson gson = null;
	QueryRunner queryRunner = null;
	GeneralPath miyunArea = null;

	public SpiderForBaidu() {
		try {
			conn = DBManager.getConnection();
			gson = new Gson();
			queryRunner = new QueryRunner();
			
			//划密云 边界
			String miyun = "117.512283, 40.673448;117.501618, 40.68091;117.486462, 40.683865;117.471233, 40.679811;117.427458, 40.691901;117.413032, 40.691963;117.368422, 40.681015;117.352677, 40.680053;117.343188, 40.672822;117.343163, 40.669474;117.332062, 40.666479;117.298595, 40.665641;117.284975, 40.672219;117.267992, 40.687306;117.248559, 40.682887;117.237938, 40.691027;117.21279, 40.700633;117.189813, 40.703272;117.185509, 40.699216;117.173492, 40.704884;117.159189, 40.703082;117.123309, 40.706232;117.11851, 40.713341;117.090641, 40.708002;117.056273, 40.706007;117.037346, 40.698128;117.010825, 40.702995;116.971399, 40.715332;116.972824, 40.719631;116.951579, 40.733725;116.947612, 40.745358;116.933105, 40.750454;116.929846, 40.757856;116.934815, 40.762214;116.930179, 40.779446;116.904581, 40.783162;116.901081, 40.788008;116.9026, 40.800071;116.892819, 40.807389;116.887238, 40.803135;116.882249, 40.805333;116.876437, 40.799851;116.870623, 40.799778;116.871529, 40.789728;116.857797, 40.784844;116.85603, 40.780956;116.843309, 40.777548;116.84634, 40.765655;116.835059, 40.754111;116.823815, 40.75567;116.813229, 40.751496;116.805806, 40.751674;116.791496, 40.762127;116.788224, 40.755495;116.796235, 40.738739;116.796839, 40.71939;116.792126, 40.706499;116.781546, 40.705204;116.766921, 40.71105;116.76003, 40.709983;116.751547, 40.701345;116.730017, 40.693604;116.72919, 40.689109;116.718, 40.681266;116.716271, 40.647531;116.707835, 40.637745;116.710412, 40.625667;116.705528, 40.625417;116.703822, 40.62135;116.712468, 40.61371;116.710662, 40.609072;116.714929, 40.605908;116.714273, 40.596008;116.720202, 40.580826;116.718294, 40.57577;116.711245, 40.568565;116.699067, 40.569968;116.686433, 40.567713;116.682552, 40.562009;116.671719, 40.56152;116.67046, 40.558983;116.67235, 40.555751;116.67633, 40.559361;116.68304, 40.559246;116.683564, 40.556545;116.706682, 40.55103;116.721253, 40.530522;116.71629, 40.521352;116.707155, 40.51423;116.702595, 40.505124;116.703866, 40.498408;116.698863, 40.495713;116.699784, 40.49128;116.708481, 40.484663;116.698234, 40.485461;116.700187, 40.473643;116.704978, 40.473149;116.715691, 40.463376;116.721236, 40.463242;116.722115, 40.465609;116.727828, 40.46281;116.725277, 40.459191;116.731158, 40.441763;116.725833, 40.445195;116.723285, 40.443547;116.729092, 40.438527;116.73056, 40.422553;116.740701, 40.424673;116.739669, 40.420977;116.747853, 40.419313;116.730366, 40.413127;116.729365, 40.409596;116.721357, 40.405423;116.715549, 40.407609;116.718878, 40.402838;116.712541, 40.396794;116.720417, 40.397849;116.718189, 40.389762;116.721101, 40.388201;116.714248, 40.379383;116.726405, 40.374672;116.724918, 40.368343;116.732577, 40.367401;116.732222, 40.362567;116.735172, 40.361307;116.730753, 40.345501;116.737369, 40.345506;116.737939, 40.341587;116.745826, 40.344165;116.753096, 40.338463;116.756179, 40.339547;116.776517, 40.32141;116.75203, 40.303117;116.74475, 40.290212;116.745041, 40.284704;116.761885, 40.281302;116.771471, 40.274564;116.774699, 40.276646;116.775956, 40.273128;116.785965, 40.280989;116.790048, 40.279782;116.795572, 40.296397;116.798597, 40.293611;116.804281, 40.296565;116.813371, 40.293192;116.817826, 40.295504;116.818883, 40.288456;116.827035, 40.287319;116.825103, 40.290767;116.830636, 40.291992;116.83559, 40.311502;116.843873, 40.316714;116.852973, 40.318151;116.857106, 40.315675;116.864002, 40.299136;116.877413, 40.297011;116.87868, 40.285675;116.882844, 40.281232;116.881071, 40.274477;116.892234, 40.262475;116.899967, 40.246834;116.907313, 40.244683;116.900951, 40.238846;116.913321, 40.234789;116.913749, 40.229531;116.928113, 40.226902;116.931841, 40.233121;116.948507, 40.238066;116.953899, 40.242563;116.958044, 40.242922;116.963395, 40.238065;116.981706, 40.253792;116.957167, 40.263336;116.9585, 40.267715;116.970782, 40.281096;116.976979, 40.282491;116.977992, 40.288182;116.995789, 40.292626;116.997078, 40.29615;117.006391, 40.296795;117.008022, 40.305841;117.015933, 40.312512;117.011752, 40.323393;117.015917, 40.331673;117.021191, 40.333186;117.025103, 40.340508;117.042467, 40.352515;117.059528, 40.349086;117.065225, 40.342551;117.078097, 40.347411;117.104539, 40.365744;117.113802, 40.36414;117.118639, 40.358967;117.122999, 40.359552;117.146128, 40.36741;117.151549, 40.374347;117.162837, 40.379896;117.169536, 40.37756;117.189117, 40.383831;117.210265, 40.379297;117.221831, 40.383987;117.230104, 40.381682;117.235062, 40.392322;117.241661, 40.395752;117.243127, 40.400548;117.246722, 40.40068;117.24604, 40.405045;117.238857, 40.410209;117.240581, 40.42306;117.256385, 40.435421;117.263364, 40.436067;117.26993, 40.447321;117.241196, 40.464298;117.242267, 40.475311;117.232341, 40.484311;117.234676, 40.48752;117.223574, 40.501212;117.215835, 40.502724;117.21653, 40.512643;117.220225, 40.518656;117.239555, 40.519506;117.245336, 40.523751;117.253813, 40.518445;117.266702, 40.525486;117.256848, 40.54366;117.259649, 40.547985;117.256124, 40.554638;117.272602, 40.564079;117.302711, 40.572713;117.317146, 40.582207;117.340905, 40.582418;117.356706, 40.586185;117.372601, 40.582034;117.382658, 40.572888;117.395929, 40.567374;117.410574, 40.579774;117.426571, 40.574504;117.436606, 40.584657;117.42995, 40.58783;117.420328, 40.612091;117.429118, 40.643382;117.478347, 40.653361;117.485398, 40.64184;117.507621, 40.642461;117.509516, 40.659031;117.513171, 40.660949;117.512283, 40.673448";
			List<Point2D.Double> polygon = new ArrayList<Point2D.Double>();
			for(String pointStr : miyun.split(";")){
				String[] pointArray =  pointStr.split(",");
				try{
				Point2D.Double point = new Point2D.Double(Double.valueOf(pointArray[0].trim()),Double.valueOf(pointArray[1].trim()));
				polygon.add(point);
				}catch(Exception e){
					System.out.println(pointArray.toString());
					e.printStackTrace();
				}
			}
			miyunArea = new GeneralPath();
			Point2D.Double first = polygon.get(0);
			miyunArea.moveTo(first.x, first.y);
			for (Point2D.Double d : polygon) {
				miyunArea.lineTo(d.x, d.y);
			}
			miyunArea.lineTo(first.x, first.y);
			miyunArea.closePath();
			
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public Set<String> keySetCache = new TreeSet<String>();
	Map<String, Boolean> typeMapCache = new Hashtable<String, Boolean>();

	/**
	 * 按照关键字进行搜索，如果bounds有值，则按照范围搜索，如果无值，则按照密云范围进行搜索
	 * @param keyword	关键字
	 * @param bounds	范围	38.76623,116.43213,39.54321,116.46773 lat,lng(左下角坐标),lat,lng(右上角坐标)
	 */
	public void search(String keyword,double[] bounds) {
		String url = "http://api.map.baidu.com/place/v2/search?ak=wsyHLdxcLR0bNu2NxYBrlN0l&output=json&scope=2&page_size=20";
		String querySql = "select id from sf_seller_baidu";
		String insertSql = "insert into SF_SELLER_BAIDU "
				+ "(ID, NAME, LAT, LNG, ADDRESS, STREET_ID, TELEPHONE, DISTANCE, TAG, TYPE, DETAIL_URL, PRICE, SHOP_HOURS, OVERALL_RATING, TASTE_RATING, SERVICE_RATING, ENVIRONMENT_RATING, FACILITY_RATING, HYGIENE_RATING, TECHNOLOGY_RATING, IMAGE_NUM, GROUPON_NUM, DISCOUNT_NUM, COMMENT_NUM, FAVORITE_NUM, CHECKIN_NUM, DI_REVIEW_KEYWORD, KEYWORD)"
				+ "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

		List<Object[]> ids;
		try {
			ids = queryRunner.query(conn, querySql, new ArrayListHandler());
			for (Object[] id : ids) {
				keySetCache.add((String) id[0]);
			}
		} catch (SQLException e1) {
			e1.printStackTrace();
		}
		try {
			int i = 0;
			int inertCount = 0;
			int existCount = 0;
			int offsideCount = 0;
			int noLocation = 0;
			int count = 0;
			int total = 0;
			Map<String,String> paramMap = new HashMap<String,String>();
			paramMap.put("query", keyword);
			//如果bound有值，则为按范围搜索
			if(bounds!=null && bounds.length == 4){
				paramMap.put("bounds", bounds[0]+","+bounds[1]+","+bounds[2]+","+bounds[3]);
				logger.info("按边界查询'" + keyword + "'，"+bounds[0]+","+bounds[1]+","+bounds[2]+","+bounds[3]);
			}else{
				paramMap.put("region", "密云");
				logger.info("按地区查询'" + keyword);
			}
			while (true) {
				paramMap.put("page_num", String.valueOf(i++));
				String rsStr = Jsoup.connect(url).data(paramMap).execute().body();
				Map<String, Object> map = gson.fromJson(rsStr, Map.class);
				if ("ok".equals(map.get("message")) && (Double) map.get("status") == 0.0) {
					int t = ((Double) map.get("total")).intValue();
					
					//如果总数超过750，则进行bound搜索
					if(t > 750){
						if(bounds == null){
							bounds = new double[]{40.212711,116.639033,40.809497,117.556598};	//密云默认最远范围
						}
						double[] b = bounds;
						
						/*
						 * b[2],b[1]					b[2],b[1]+(b[3]-b[1])/2						b[2],b[3]
						 * 
						 * b[0]+(b[2]-b[0])/2,b[1]		b[0]+(b[2]-b[0])/2,b[1]+(b[3]-b[1])/2		b[0]+(b[2]-b[0])/2,b[3]
						 * 
						 * b[0],b[1]					b[0],b[1]+(b[3]-b[1])/2						b[0],b[3]
						 */
						search(keyword,new double[]{b[0],b[1],b[0]+(b[2]-b[0])/2,b[1]+(b[3]-b[1])/2});	//左下
						search(keyword,new double[]{b[0]+(b[2]-b[0])/2,b[1],b[2],b[1]+(b[3]-b[1])/2});	//左上
						search(keyword,new double[]{b[0],b[1]+(b[3]-b[1])/2,b[0]+(b[2]-b[0])/2,b[3]});	//右下
						search(keyword,new double[]{b[0]+(b[2]-b[0])/2,b[1]+(b[3]-b[1])/2,b[2],b[3]});	//右上
						break;
					}
					
					if (t > 0) {
						total = total > t ? total : t;
						if (map.containsKey("results")) {
							List<Map<String, Object>> resultsList = (List<Map<String, Object>>) map.get("results");
							for (Map<String, Object> result : resultsList) {
								++count;
								Map<String, Object> location = (Map<String, Object>) result.get("location");
								if(location == null || location.get("lat") == null || location.get("lng") == null){
									++noLocation;
									continue;
								}
								Map<String, Object> detail_info = (Map<String, Object>) result.get("detail_info");
								if (detail_info == null) {
									detail_info = new HashMap<String, Object>();
								}
								//判断是否在边界范围内
								if(!isIn((Double) location.get("lat"),(Double) location.get("lng"))){
									++offsideCount;
									continue;
								}
								
								String uid = (String) result.get("uid");
								if (uid != null) {
									try {
										if (keySetCache.contains(uid)) {
											++existCount;
										} else {
											List<Object> paramsList = new ArrayList<Object>();
											paramsList.add(uid);
											paramsList.add((String) result.get("name"));
											paramsList.add(((Double) location.get("lat")).toString());
											paramsList.add(((Double) location.get("lng")).toString());
											paramsList.add((String) result.get("address"));
											paramsList.add((String) result.get("street_id"));
											paramsList.add((String) result.get("telephone"));
											paramsList.add(detail_info.get("distance") == null ? null : String
													.valueOf(detail_info.get("distance")));
											paramsList.add((String) detail_info.get("type"));
											paramsList.add((String) detail_info.get("tag"));
											paramsList.add((String) detail_info.get("detail_url"));
											paramsList.add((String) detail_info.get("price"));
											paramsList.add((String) detail_info.get("shop_hours"));
											paramsList.add((String) detail_info.get("overall_rating"));
											paramsList.add((String) detail_info.get("taste_rating"));
											paramsList.add((String) detail_info.get("service_rating"));
											paramsList.add((String) detail_info.get("environment_rating"));
											paramsList.add((String) detail_info.get("facility_rating"));
											paramsList.add((String) detail_info.get("hygiene_rating"));
											paramsList.add((String) detail_info.get("technology_rating"));
											paramsList.add((String) detail_info.get("image_num"));
											paramsList.add(detail_info.get("groupon_num") == null ? null : String
													.valueOf(detail_info.get("groupon_num")));
											paramsList.add(detail_info.get("discount_num") == null ? null : String
													.valueOf(detail_info.get("discount_num")));
											paramsList.add((String) detail_info.get("comment_num"));
											paramsList.add((String) detail_info.get("favorite_num"));
											paramsList.add((String) detail_info.get("checkin_num"));
											String di_review_keyword = null;
											if (detail_info.containsKey("di_review_keyword")) {
												di_review_keyword = gson.toJson(detail_info.get("di_review_keyword"));
											}
											paramsList.add(di_review_keyword);
											paramsList.add(keyword);
											// queryRunner.update(conn,
											// insertSql, paramsList.toArray());
											queryRunner.update(conn, insertSql, paramsList.toArray());
											keySetCache.add(uid);
											++inertCount;
										}
									} catch (SQLException e) {
										e.printStackTrace();
									}
									
									String tag = (String) detail_info.get("tag");
									if (tag != null && !"".equals(tag)) {
										for (String word : tag.split(";")) {
											if (word != null && !"".equals(word)) {
												if (!typeMapCache.containsKey(word)) {
													typeMapCache.put(word, true);
													logger.info("加入新关键字'" + word + "'");
													logger.info(gson.toJson(typeMapCache));
												}
											}
										}
									}
									
									String type = (String) detail_info.get("type");
									if (type != null && !"".equals(type)) {
										for (String word : type.split(";")) {
											if (word != null && !"".equals(word)) {
												if (!typeMapCache.containsKey(word)) {
													typeMapCache.put(word, true);
													logger.info("加入新关键字'" + word + "'");
													logger.info(gson.toJson(typeMapCache));
												}
											}
										}
									}
								}
							}
						}
					} else {
						logger.info("轮询完成，共计" + (i + 1) + "页！");
						break;
					}
				} else {
					logger.error("message:" + map.get("message") + "\tstatus" + map.get("status"));
					logger.info("非正常结束！");
					break;
				}
			}
			logger.info("报告：关键字'" + keyword + "'，标称" + total + "条，轮询" + count + "条，插入" + inertCount + "条，跳过"
					+ existCount + "条，超出边界" + offsideCount + "条，信息不全" + noLocation + "条");
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	
	private boolean isIn(double lat,double lng){
		return this.miyunArea.contains(lng,lat);
	}

	public static void main(String[] args) {
		
		SpiderForBaidu spider = new SpiderForBaidu();
		//String[] keywords = new String[] { "美食", "购物", "酒店", "休闲娱乐", "交通设施", "金融", "旅游景点", "政府机构", "生活服务", "丽人",
		//		"公司企业", "运动健身", "教育培训", "房地产", "医疗", };
		String[] keywords = new String[] { "美食", "购物", "酒店", "休闲娱乐", "交通设施", "金融", "旅游景点", "政府机构", "生活服务", "丽人",
				"公司企业", "运动健身", "教育培训", "房地产", "医疗", };
		for (String key : keywords) {
			spider.typeMapCache.put(key, true);
		}
		Gson gson = new Gson();
		while (true) {
			boolean exit = true;
			Map<String, Boolean> typeMap = new HashMap<String, Boolean>();
			typeMap.putAll(spider.typeMapCache);
			logger.info(gson.toJson(typeMap));
			Iterator<String> iter = typeMap.keySet().iterator();
			while (iter.hasNext()) {
				String key = iter.next();
				if (typeMap.get(key)) {
					exit = false;
					logger.info("开始搜索关键字：" + key);
					spider.search(key,null);
					spider.typeMapCache.put(key, false);
				}
			}
			if(exit){
				logger.info("搜索完成！");
				break;
			}
				
		}
	}
}
